{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "1106.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/1106.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yz6zaiMpSmrE",
        "colab_type": "text"
      },
      "source": [
        "## 4.4 自定义层"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnsCE-gMS6eT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install mxnet d2lzh"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wtmt1U_O09a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import gluon, nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "class CenteredLayer(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(CenteredLayer, self).__init__(**kwargs)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return x - x.mean()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrGLKTCqP9cW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "969d499c-7459-447a-9e20-fa1244c9777e"
      },
      "source": [
        "layer = CenteredLayer()\n",
        "layer(nd.array([1, 2, 3, 4, 5]))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[-2. -1.  0.  1.  2.]\n",
              "<NDArray 5 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cupObmAvQGmt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(128), \n",
        "    CenteredLayer())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfforiFLa4zY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "40dde463-0dc0-4ada-fa0b-56d7d32b6416"
      },
      "source": [
        "net.initialize()\n",
        "y = net(nd.random.uniform(shape=(4, 8)))\n",
        "y.mean().asscalar()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-7.212293e-10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VCF_-rrJbFOg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "7a9fcaee-8de0-4486-fb97-af4e66f85224"
      },
      "source": [
        "params = gluon.ParameterDict()\n",
        "params.get('param2', shape=(2, 3))\n",
        "params"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\n",
              "  Parameter param2 (shape=(2, 3), dtype=<class 'numpy.float32'>)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydokxsIcbmVP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyDense(nn.Block):\n",
        "    def __init__(self, units, in_units, **kwargs):\n",
        "        super(MyDense, self).__init__(**kwargs)\n",
        "        self.weight = self.params.get('weight', shape=(in_units, units))\n",
        "        self.bias = self.params.get('bias', shape=(units, ))\n",
        "\n",
        "    def forward(self, x):\n",
        "        linear = nd.dot(x, self.weight.data()) + self.bias.data()\n",
        "        return nd.relu(linear)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydllsSPUc5t_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "98d894de-f7a2-4ad9-ffd2-a9b0a60f913b"
      },
      "source": [
        "dense = MyDense(units=3, in_units=5)\n",
        "dense.params "
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "mydense0_ (\n",
              "  Parameter mydense0_weight (shape=(5, 3), dtype=<class 'numpy.float32'>)\n",
              "  Parameter mydense0_bias (shape=(3,), dtype=<class 'numpy.float32'>)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cqP0G3XdAzg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "bb10f9c0-f74f-4d52-9c1e-4b8e52204f18"
      },
      "source": [
        "dense.initialize()\n",
        "dense(nd.random.uniform(shape=(2, 5)))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.06917784 0.01627153 0.01029644]\n",
              " [0.02602214 0.0453731  0.        ]]\n",
              "<NDArray 2x3 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fpnij8DgdKhQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "b6adb653-59ed-41b9-c86c-cbcf8169c16d"
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(MyDense(8, in_units=64), \n",
        "    MyDense(1, in_units=8))\n",
        "net.initialize()\n",
        "net(nd.random.uniform(shape=(2, 64)))\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.03820474]\n",
              " [0.04035058]]\n",
              "<NDArray 2x1 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H71g_EeJdnqh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "x = nd.ones(3)\n",
        "nd.save('x', x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ys1JvT7BeBan",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "205cfbdb-1f68-4730-8576-43cafbadc253"
      },
      "source": [
        "x2 = nd.load('x')\n",
        "\n",
        "x2"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOxFz7rYeHEA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "828c985c-12f5-4013-9f29-449b8b2ad892"
      },
      "source": [
        "y = nd.zeros(4)\n",
        "nd.save('xy', [x, y])\n",
        "x2, y2 = nd.load('xy')\n",
        "(x2, y2)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>, \n",
              " [0. 0. 0. 0.]\n",
              " <NDArray 4 @cpu(0)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DvZWxBsxjmx6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "77d14f64-838d-4522-8521-e0b6f4aa95c3"
      },
      "source": [
        "mydict = {'x': x, 'y': y}\n",
        "nd.save('mydict', mydict)\n",
        "mydict2 = nd.load('mydict')\n",
        "mydict2"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'x': \n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>, 'y': \n",
              " [0. 0. 0. 0.]\n",
              " <NDArray 4 @cpu(0)>}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ms2n40gNj_so",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(MLP, self).__init__(**kwargs)\n",
        "        self.hidden = nn.Dense(256, activation='relu')\n",
        "        self.output = nn.Dense(10)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        return self.output(self.hidden(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mw-FCz0an7KH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(MLP, self).__init__(**kwargs)\n",
        "        self.hidden = nn.Dense(256, activation='relu')\n",
        "        self.output = nn.Dense(10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.output(self.hidden(x))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdTsR3a3kyHw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = MLP()\n",
        "\n",
        "net.initialize()\n",
        "X = nd.random.uniform(shape=(2, 20))\n",
        "Y = net(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9mNs_PJbk7bs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filename = 'mlp.params'\n",
        "net.save_parameters(filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8VVdztXclEQZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net2 = MLP()\n",
        "net2.load_parameters(filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yYsWMCdQmXSB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "7a7a88a0-a285-44e9-fa22-84e9cf4e3653"
      },
      "source": [
        "Y2 = net2(X)\n",
        "Y2 == Y"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n",
              " [1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]]\n",
              "<NDArray 2x10 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "shebCYrembRK",
        "colab_type": "text"
      },
      "source": [
        "## 4.6 GPU 计算"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bgWPlE2Rp0JC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "outputId": "04b2f60f-176a-44eb-b194-e251b71851b5"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Nov  6 07:39:48 2019       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 430.50       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TAGFmilAp4fS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f363a947-ea30-4916-a913-d80adff72d3b"
      },
      "source": [
        "import mxnet as mx \n",
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "mx.cpu(), mx.gpu()"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(cpu(0), gpu(0))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2U34dtSOqX7z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "84866661-84fc-448c-e903-1b6e93102031"
      },
      "source": [
        "mx.gpu(1)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "gpu(1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1iLEyfOiqf9B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = nd.array([1, 2, 3])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GKnHg9IKqoCb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9556858b-b2bc-4662-e150-a6832f5fda72"
      },
      "source": [
        "x.context"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "cpu(0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ILRS_luqqyS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = nd.array([1, 2, 3], ctx=mx.gpu())\n",
        "a"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BDH7CvBIq5sJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "B = nd.random.uniform(shape=(2, 3), ctx=mx.gpu(1))\n",
        "B"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPm77xwHrT4y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = x.copyto(mx.gpu())\n",
        "y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_cju5zfrtsu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "z = x.as_in_context(mx.gpu)\n",
        "z"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BAKSJP5cr1Qh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y.as_in_context(mx.gpu()) is y "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eXn2uPfjr_Gk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y.copyto(mx.gpu()) is y "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cmv2r4C4sGgP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "(z + 2).exp() * y "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m75c4yybsMsC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(1))\n",
        "net.initialize(ctx=mx.gpnet(yu)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MeyM8AnOsYlk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net[0].weight.data()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}